{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, re, copy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import json, gzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = os.path.expanduser(\"~/blob/raw_datasets/amazon-beauty-2014\")\n",
    "DATASET_NAME = \"Beauty\"\n",
    "\n",
    "ALL_AMAZON_PROD = os.path.expanduser(\"~/blob/raw_datasets/amazon-all/All_Amazon_Meta.json.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load from tsv file.\n",
      "Columns of reviews:  Index(['reviewerID', 'asin', 'reviewerName', 'helpful', 'reviewText',\n",
      "       'overall', 'summary', 'unixReviewTime', 'reviewTime'],\n",
      "      dtype='object')\n",
      "Columns of meta data:  Index(['asin', 'description', 'title', 'imUrl', 'salesRank', 'categories',\n",
      "       'price', 'related', 'brand'],\n",
      "      dtype='object')\n",
      "Shape of reviews:  (2023070, 9)\n",
      "Shape of metas:  (259204, 9)\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def parse(path):\n",
    "    g = gzip.open(path, 'rb')\n",
    "    for l in g:\n",
    "        p = re.compile('(?<!\\\\\\\\)\\'')\n",
    "        l = p.sub('\\\"', l)\n",
    "        yield json.loads(l)\n",
    "\n",
    "def getDF(path):\n",
    "    i = 0\n",
    "    df = {}\n",
    "    for d in parse(path):\n",
    "        df[i] = d\n",
    "        i += 1\n",
    "    return pd.DataFrame.from_dict(df, orient='index')\n",
    "\n",
    "\n",
    "def load_all_amazon_product_meta(path):\n",
    "    return getDF(path)\n",
    "\n",
    "\n",
    "if (not os.path.exists(os.path.join(DATA_DIR, \"reviews.tsv\"))) or (not os.path.exists(os.path.join(DATA_DIR, \"meta.tsv\"))) :\n",
    "    print(\"Load from json file and save as tsv file.\")\n",
    "    review_df = getDF(os.path.join(DATA_DIR, f'reviews_{DATASET_NAME}.json.gz'))\n",
    "    meta_df = getDF(os.path.join(DATA_DIR, f'meta_{DATASET_NAME}.json.gz'))\n",
    "    review_df.to_csv(os.path.join(DATA_DIR, \"reviews.tsv\"), index=None)\n",
    "    meta_df.to_csv(os.path.join(DATA_DIR, \"meta.tsv\"), index=None)\n",
    "else:\n",
    "    print(\"Load from tsv file.\")\n",
    "    review_df = pd.read_csv(os.path.join(DATA_DIR, \"reviews.tsv\"), low_memory=False)\n",
    "    meta_df = pd.read_csv(os.path.join(DATA_DIR, \"meta.tsv\"), low_memory=False)\n",
    "    \n",
    "print(\"Columns of reviews: \", review_df.columns)\n",
    "print(\"Columns of meta data: \", meta_df.columns)\n",
    "print(\"Shape of reviews: \", review_df.shape)\n",
    "print(\"Shape of metas: \", meta_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2023070, 9), (259204, 9))"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_df.shape, meta_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>imUrl</th>\n",
       "      <th>salesRank</th>\n",
       "      <th>categories</th>\n",
       "      <th>price</th>\n",
       "      <th>related</th>\n",
       "      <th>brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3669</th>\n",
       "      <td>B0001DHH0C</td>\n",
       "      <td>* 7\" Style Comb with All Fine Teeth * Color: B...</td>\n",
       "      <td>Ace All-purpose Comb 7&amp;quot; Fine-teeth * Black</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/310G18Qd...</td>\n",
       "      <td>{'Beauty': 39222}</td>\n",
       "      <td>[['Beauty', 'Hair Care', 'Styling Tools', 'Com...</td>\n",
       "      <td>4.99</td>\n",
       "      <td>{'also_bought': ['B00B7RHYTK', 'B00DAJW0R4', '...</td>\n",
       "      <td>ACE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146045</th>\n",
       "      <td>B005HKEEFK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mally Beauty Evercolor Endless Eyeshadow (Twil...</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/311uUeG5...</td>\n",
       "      <td>{'Beauty': 423529}</td>\n",
       "      <td>[['Beauty', 'Makeup', 'Eyes', 'Eye Shadow']]</td>\n",
       "      <td>18.00</td>\n",
       "      <td>{'also_bought': ['B003JLOSTG', 'B007TN0RDQ', '...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43617</th>\n",
       "      <td>B0011DPWT4</td>\n",
       "      <td>Bath &amp; Body Works C.O. Bigelow Rosemary Mint S...</td>\n",
       "      <td>Bath &amp;amp; Body Works C.O. Bigelow Rosemary Mi...</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41TNk65l...</td>\n",
       "      <td>{'Beauty': 849088}</td>\n",
       "      <td>[['Beauty', 'Bath &amp; Body', 'Cleansers', 'Body ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236613</th>\n",
       "      <td>B00EKZ3T3Q</td>\n",
       "      <td>Renowned fashion illustrator Antonio Lopez col...</td>\n",
       "      <td>MAC Antonio Lopez Face/pink Palette</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/4114jpFa...</td>\n",
       "      <td>{'Beauty': 371488}</td>\n",
       "      <td>[['Beauty', 'Makeup', 'Face', 'Blush']]</td>\n",
       "      <td>52.40</td>\n",
       "      <td>{'also_viewed': ['B00HKA110G']}</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53531</th>\n",
       "      <td>B001A9GMSY</td>\n",
       "      <td>Bath &amp; Body Works Cucumber Melon Body Cream 8....</td>\n",
       "      <td>Bath &amp;amp; Body Works Cucumber Melon Body Crea...</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/21AHFko9...</td>\n",
       "      <td>{'Beauty': 327917}</td>\n",
       "      <td>[['Beauty', 'Skin Care', 'Body', 'Moisturizers...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'also_bought': ['B002G7H1IY', 'B003Q2F2RA'], ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              asin                                        description  \\\n",
       "3669    B0001DHH0C  * 7\" Style Comb with All Fine Teeth * Color: B...   \n",
       "146045  B005HKEEFK                                                NaN   \n",
       "43617   B0011DPWT4  Bath & Body Works C.O. Bigelow Rosemary Mint S...   \n",
       "236613  B00EKZ3T3Q  Renowned fashion illustrator Antonio Lopez col...   \n",
       "53531   B001A9GMSY  Bath & Body Works Cucumber Melon Body Cream 8....   \n",
       "\n",
       "                                                    title  \\\n",
       "3669      Ace All-purpose Comb 7&quot; Fine-teeth * Black   \n",
       "146045  Mally Beauty Evercolor Endless Eyeshadow (Twil...   \n",
       "43617   Bath &amp; Body Works C.O. Bigelow Rosemary Mi...   \n",
       "236613                MAC Antonio Lopez Face/pink Palette   \n",
       "53531   Bath &amp; Body Works Cucumber Melon Body Crea...   \n",
       "\n",
       "                                                    imUrl           salesRank  \\\n",
       "3669    http://ecx.images-amazon.com/images/I/310G18Qd...   {'Beauty': 39222}   \n",
       "146045  http://ecx.images-amazon.com/images/I/311uUeG5...  {'Beauty': 423529}   \n",
       "43617   http://ecx.images-amazon.com/images/I/41TNk65l...  {'Beauty': 849088}   \n",
       "236613  http://ecx.images-amazon.com/images/I/4114jpFa...  {'Beauty': 371488}   \n",
       "53531   http://ecx.images-amazon.com/images/I/21AHFko9...  {'Beauty': 327917}   \n",
       "\n",
       "                                               categories  price  \\\n",
       "3669    [['Beauty', 'Hair Care', 'Styling Tools', 'Com...   4.99   \n",
       "146045       [['Beauty', 'Makeup', 'Eyes', 'Eye Shadow']]  18.00   \n",
       "43617   [['Beauty', 'Bath & Body', 'Cleansers', 'Body ...    NaN   \n",
       "236613            [['Beauty', 'Makeup', 'Face', 'Blush']]  52.40   \n",
       "53531   [['Beauty', 'Skin Care', 'Body', 'Moisturizers...    NaN   \n",
       "\n",
       "                                                  related brand  \n",
       "3669    {'also_bought': ['B00B7RHYTK', 'B00DAJW0R4', '...   ACE  \n",
       "146045  {'also_bought': ['B003JLOSTG', 'B007TN0RDQ', '...   NaN  \n",
       "43617                                                 NaN   NaN  \n",
       "236613                    {'also_viewed': ['B00HKA110G']}   NaN  \n",
       "53531   {'also_bought': ['B002G7H1IY', 'B003Q2F2RA'], ...   NaN  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_df['categories'] = meta_df['categories'].apply(lambda x: eval(x)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(258760, 10)\n"
     ]
    }
   ],
   "source": [
    "meta_df = meta_df[~meta_df['title'].isna()]\n",
    "meta_df.reset_index(inplace=True)\n",
    "print(meta_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>helpful</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>overall</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>reviewTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>437374</th>\n",
       "      <td>AA0BJD80IPGU9</td>\n",
       "      <td>B000V26BQY</td>\n",
       "      <td>George C. Reeves</td>\n",
       "      <td>[0, 0]</td>\n",
       "      <td>I purchased this item for my wife, and she abs...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Clarisonic</td>\n",
       "      <td>1310601600</td>\n",
       "      <td>07 14, 2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1720292</th>\n",
       "      <td>A18OEWMZXSJ826</td>\n",
       "      <td>B0096DS22U</td>\n",
       "      <td>Hamada</td>\n",
       "      <td>[0, 0]</td>\n",
       "      <td>I like Fendi too  muchGood priceI like it</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Nice</td>\n",
       "      <td>1405036800</td>\n",
       "      <td>07 11, 2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796403</th>\n",
       "      <td>AY7VU86VFTU27</td>\n",
       "      <td>B001V6QL5K</td>\n",
       "      <td>tahisha hall</td>\n",
       "      <td>[0, 0]</td>\n",
       "      <td>This product is a must if you are trying to ma...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Natural hair crack!</td>\n",
       "      <td>1396569600</td>\n",
       "      <td>04 4, 2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1573586</th>\n",
       "      <td>A1XQRGA4MG5VW3</td>\n",
       "      <td>B007EF59RS</td>\n",
       "      <td>Susan L. Schwartz</td>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>I've been using Shalimar since high school (a ...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Love Shalimar</td>\n",
       "      <td>1361577600</td>\n",
       "      <td>02 23, 2013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1037870</th>\n",
       "      <td>A26EHQ52RL9LL1</td>\n",
       "      <td>B003F64FN4</td>\n",
       "      <td>Eric Zuniga</td>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>Good hold, especially for thick hair without l...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Great Product</td>\n",
       "      <td>1391644800</td>\n",
       "      <td>02 6, 2014</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             reviewerID        asin       reviewerName helpful  \\\n",
       "437374    AA0BJD80IPGU9  B000V26BQY   George C. Reeves  [0, 0]   \n",
       "1720292  A18OEWMZXSJ826  B0096DS22U             Hamada  [0, 0]   \n",
       "796403    AY7VU86VFTU27  B001V6QL5K       tahisha hall  [0, 0]   \n",
       "1573586  A1XQRGA4MG5VW3  B007EF59RS  Susan L. Schwartz  [1, 1]   \n",
       "1037870  A26EHQ52RL9LL1  B003F64FN4        Eric Zuniga  [1, 1]   \n",
       "\n",
       "                                                reviewText  overall  \\\n",
       "437374   I purchased this item for my wife, and she abs...      5.0   \n",
       "1720292          I like Fendi too  muchGood priceI like it      4.0   \n",
       "796403   This product is a must if you are trying to ma...      5.0   \n",
       "1573586  I've been using Shalimar since high school (a ...      5.0   \n",
       "1037870  Good hold, especially for thick hair without l...      5.0   \n",
       "\n",
       "                     summary  unixReviewTime   reviewTime  \n",
       "437374            Clarisonic      1310601600  07 14, 2011  \n",
       "1720292                 Nice      1405036800  07 11, 2014  \n",
       "796403   Natural hair crack!      1396569600   04 4, 2014  \n",
       "1573586        Love Shalimar      1361577600  02 23, 2013  \n",
       "1037870        Great Product      1391644800   02 6, 2014  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "used_col = {\n",
    "    'review': ['reviewerID', 'asin', 'overall', 'unixReviewTime'],\n",
    "    'meta': ['asin', 'title', 'categories', 'price', 'description', 'brand'],\n",
    "    }\n",
    "review_df = review_df[used_col['review']]\n",
    "meta_df = meta_df[used_col['meta']]\n",
    "\n",
    "review_df = review_df.rename(columns={'overall': 'rating', 'unixReviewTime': 'timestamp', 'asin': 'item_id', 'reviewerID': 'user_id'})\n",
    "meta_df = meta_df.rename(columns={'asin': 'item_id', 'categories': 'category'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          user_id     item_id  rating   timestamp\n",
      "0  A39HTATAQ9V7YF  0205616461     5.0  1369699200\n",
      "1  A3JM6GV9MNOF9X  0558925278     3.0  1355443200\n",
      "2  A1Z513UWSAAO0F  0558925278     5.0  1404691200\n",
      "      item_id                                              title  \\\n",
      "0  0205616461  Bio-Active Anti-Aging Serum (Firming Ultra-Hyd...   \n",
      "1  0558925278  Eco Friendly Ecotools Quality Natural Bamboo C...   \n",
      "2  0733001998                                Mastiha Body Lotion   \n",
      "\n",
      "                                            category  price  \\\n",
      "0   [Beauty, Skin Care, Face, Creams & Moisturizers]    NaN   \n",
      "1  [Beauty, Tools & Accessories, Makeup Brushes &...    NaN   \n",
      "2   [Beauty, Skin Care, Body, Moisturizers, Lotions]    NaN   \n",
      "\n",
      "                                         description brand  \n",
      "0  As we age, our once youthful, healthy skin suc...   NaN  \n",
      "1  Mineral Powder Brush--Apply powder or mineral ...   NaN  \n",
      "2  From the Greek island of Chios, this Mastiha b...   NaN  \n"
     ]
    }
   ],
   "source": [
    "print(review_df.head(3))\n",
    "print(meta_df.head(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_valid_ids(df, col_name, k):\n",
    "        frequency = df.groupby([col_name])[[col_name]].count()\n",
    "        valid_id = frequency[frequency[col_name]>=k].index\n",
    "        return valid_id\n",
    "\n",
    "def keepFirstFilter(df: pd.DataFrame, user_col: str='user_id', item_col: str='item_id', time_col: str='timestamp') -> pd.DataFrame:\n",
    "    print(f\"*** Only keep the first interaction for duplicated review ***\")\n",
    "    print(f\"Begin: {df.shape}\")\n",
    "    df = df.sort_values(by=[user_col, time_col], ).reset_index(drop=True)\n",
    "    df = df.drop_duplicates(subset=[user_col, item_col], keep='first').reset_index(drop=True)\n",
    "    print(f\"After: {df.shape}\")\n",
    "    return df\n",
    "\n",
    "\n",
    "def kCoreFilter(df: pd.DataFrame, user_k: int=10, item_k: int=10, user_col: str='user_id', item_col: str='item_id', max_iter: int=20) -> pd.DataFrame:\n",
    "    print(f\"*** Users whose interaction < {user_k} and items whose interaction < {item_k}  would be filtered out ***\")\n",
    "    print(f\"Begin: {df.shape}\")\n",
    "    \n",
    "    num_users_prev, num_items_prev = len(df[user_col].unique()), len(df[item_col].unique()) \n",
    "    delta = True\n",
    "    iter = 0\n",
    "    \n",
    "    while delta and iter < max_iter: \n",
    "        valid_users = get_valid_ids(df, user_col, user_k)  \n",
    "        df = df[df[user_col].isin(valid_users)]\n",
    "\n",
    "        valid_items = get_valid_ids(df, item_col, item_k)  \n",
    "        df = df[df[item_col].isin(valid_items)]\n",
    "\n",
    "        num_users = len(valid_users)\n",
    "        num_items = len(valid_items)  \n",
    "\n",
    "        delta = (num_users != num_users_prev) or (num_items != num_items_prev)\n",
    "        print('Iter: {0}, users: {1} / {2}, items: {3} / {4}'.format(iter, num_users, num_users_prev, num_items, num_items_prev))\n",
    "\n",
    "        num_users_prev = num_users\n",
    "        num_items_prev = num_items\n",
    "        iter+=1\n",
    "    print(f\"After: {df.shape}\")\n",
    "    return df \n",
    "\n",
    "\n",
    "def lowRatingFilter(df: pd.DataFrame, rating_thres: float=3.0, rating_col: str='rating') -> pd.DataFrame:\n",
    "    print(f\"*** Rating < {rating_thres} would be filtered out ***\")\n",
    "    print(f\"Begin: {df.shape}\")\n",
    "    df = df[df[rating_col] >= rating_thres].reset_index(drop=True)\n",
    "    print(f\"After: {df.shape}\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_df = review_df[review_df['item_id'].isin(meta_df['item_id'])].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Only keep the first interaction for duplicated review ***\n",
      "Begin: (2020642, 4)\n",
      "After: (2020642, 4)\n",
      "*** Rating < 3.0 would be filtered out ***\n",
      "Begin: (2020642, 4)\n",
      "After: (1724132, 4)\n",
      "*** Users whose interaction < 5 and items whose interaction < 5  would be filtered out ***\n",
      "Begin: (1724132, 4)\n",
      "Iter: 0, users: 42949 / 1047744, items: 15791 / 231472\n",
      "Iter: 1, users: 20766 / 42949, items: 10555 / 15791\n",
      "Iter: 2, users: 17269 / 20766, items: 9340 / 10555\n",
      "Iter: 3, users: 16223 / 17269, items: 8947 / 9340\n",
      "Iter: 4, users: 15852 / 16223, items: 8785 / 8947\n",
      "Iter: 5, users: 15675 / 15852, items: 8718 / 8785\n",
      "Iter: 6, users: 15625 / 15675, items: 8689 / 8718\n",
      "Iter: 7, users: 15587 / 15625, items: 8680 / 8689\n",
      "Iter: 8, users: 15577 / 15587, items: 8678 / 8680\n",
      "Iter: 9, users: 15576 / 15577, items: 8678 / 8678\n",
      "Iter: 10, users: 15576 / 15576, items: 8678 / 8678\n",
      "After: (139318, 4)\n"
     ]
    }
   ],
   "source": [
    "review_df1 = keepFirstFilter(review_df)\n",
    "review_df1 = lowRatingFilter(review_df1, 3.0)\n",
    "review_df1 = kCoreFilter(review_df1, 5, 5)\n",
    "data_df = review_df1.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                      user_id     item_id  rating   timestamp\n",
      "0       A00700212KB3K0MVESPIY  B001RMP7M6     5.0  1385337600\n",
      "1       A00700212KB3K0MVESPIY  B003TMO3EU     5.0  1385337600\n",
      "2       A00700212KB3K0MVESPIY  B00028M3N2     5.0  1386028800\n",
      "3       A00700212KB3K0MVESPIY  B0035RF85C     5.0  1386028800\n",
      "4       A00700212KB3K0MVESPIY  B006GK5NNW     5.0  1386028800\n",
      "...                       ...         ...     ...         ...\n",
      "139313          AZZT1ERHBSNQ8  B000WYZ9Q4     4.0  1405123200\n",
      "139314          AZZT1ERHBSNQ8  B003BMJGL8     5.0  1405123200\n",
      "139315          AZZT1ERHBSNQ8  B004TSFE6Y     3.0  1405123200\n",
      "139316          AZZT1ERHBSNQ8  B006ZBP8NM     5.0  1405123200\n",
      "139317          AZZT1ERHBSNQ8  B007O7AZBG     5.0  1405123200\n",
      "\n",
      "[139318 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "print(data_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_id(df: pd.DataFrame, user_colname: str='user_id', item_colname: str='item_id', group_colname: str=None, return_map: bool=False, price_df: pd.DataFrame=None, group_df: pd.DataFrame=None) -> pd.DataFrame:\n",
    "    '''map user and item ids'''\n",
    "    users = df[user_colname].unique()\n",
    "    items = df[item_colname].unique()\n",
    "    print(\"**Map users and items**\")\n",
    "    n_users, n_items = len(users), len(items)\n",
    "    user_map = {u: k+1 for k, u in enumerate(users)}\n",
    "    item_map = {i: k+1 for k, i in enumerate(items)}\n",
    "    df[user_colname] = df[user_colname].apply(lambda x: user_map[x])\n",
    "    df[item_colname] = df[item_colname].apply(lambda x: item_map[x])\n",
    "\n",
    "    if (group_colname is not None) and (group_colname in df):\n",
    "        groups = df[group_colname].unique()\n",
    "        n_groups = len(groups)\n",
    "        group_map = {g: k+1 for k,g in enumerate(groups)}\n",
    "        df[group_colname] = df[group_colname].apply(lambda x: group_map[x])\n",
    "    else:\n",
    "        group_map = {}\n",
    "\n",
    "    if price_df is not None:\n",
    "        if user_colname in price_df:\n",
    "            price_df = price_df[price_df[user_colname].isin(users)].reset_index(drop=True)\n",
    "            price_df[user_colname] = price_df.apply(lambda x: user_map[x])\n",
    "        if item_colname in price_df:\n",
    "            price_df = price_df[price_df[item_colname].isin(items)].reset_index(drop=True)\n",
    "            price_df[item_colname] = price_df[item_colname].apply(lambda x: item_map[x])\n",
    "\n",
    "    if group_df is not None:\n",
    "        if user_colname in group_df:\n",
    "            group_df = group_df[group_df[user_colname].isin(users)].reset_index(drop=True)\n",
    "            group_df[user_colname] = group_df.apply(lambda x: user_map[x])\n",
    "        if item_colname in group_df:\n",
    "            group_df = group_df[group_df[item_colname].isin(items)].reset_index(drop=True)\n",
    "            group_df[item_colname] = group_df[item_colname].apply(lambda x: item_map[x])\n",
    "        if group_colname is not None and group_colname in group_df:\n",
    "            groups = group_df[group_colname].unique()\n",
    "            n_groups = len(groups)\n",
    "            group_map = {g: k+1 for k,g in enumerate(groups)}\n",
    "            group_df[group_colname] = group_df[group_colname].apply(lambda x: group_map[x])\n",
    "\n",
    "    if return_map:\n",
    "        return (df, price_df, group_df), (user_map, item_map, group_map)\n",
    "    else:\n",
    "        return df, price_df, group_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**Map users and items**\n"
     ]
    }
   ],
   "source": [
    "(df, price_df, group_df),(user_map, item_map, group_map) = map_id(data_df, return_map=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_map = {'item': item_map, 'user': user_map}\n",
    "\n",
    "with open(os.path.join(DATA_DIR, \"map.json\"), 'w') as f:\n",
    "    json.dump(all_map, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split\n",
    "def split_train_test_set_leave_one_out_seq(data: pd.DataFrame, col_name: str, time_colname:str, col_names_2_return: list, seed: int=42):\n",
    "    '''Leave the last one item for test set'''\n",
    "    if time_colname in data:\n",
    "        df_sorted = data.sort_values(by=[col_name, time_colname]).reset_index(drop=True)\n",
    "    else:\n",
    "        df_sorted = data.sort_values(by=col_name).reset_index(drop=True)\n",
    "\n",
    "    df_test = df_sorted.groupby(by=col_name, as_index=False).nth(-1)\n",
    "    df_train = df_sorted.iloc[df_sorted.index.difference(df_test.index)]\n",
    "    return df_train.reset_index(drop=True)[col_names_2_return], df_test.reset_index(drop=True)[col_names_2_return]\n",
    "\n",
    "def split_train_test_set_leave_one_out(data: pd.DataFrame, col_name: str, col_names_2_return: list, seed: int = 0):\n",
    "    if col_names_2_return is None:\n",
    "        col_names_2_return = data.columns #.to_list()\n",
    "    df_groupby = data.groupby(by=col_name, as_index=False) \n",
    "    df_test = df_groupby.sample(n=1, random_state=seed)[col_names_2_return]\n",
    "    df_train = data.iloc[data.index.difference(df_test.index)][col_names_2_return] \n",
    "    return df_train.reset_index(drop=True), df_test.reset_index(drop=True) \n",
    "\n",
    "def split_train_test_set_by_ratio(data: pd.DataFrame, ratio: list, col_name: str, col_names_2_return: list, seed: int = 0):\n",
    "    if col_names_2_return is None:\n",
    "        col_names_2_return = data.columns #.to_list()\n",
    "    assert len(ratio) == 2, 'ratio is for train/test.'\n",
    "    frac = ratio[1] / sum(ratio)\n",
    "    df_groupby = data.groupby(by=col_name, as_index=False) \n",
    "    df_test = df_groupby.sample(frac=frac, random_state=seed)[col_names_2_return]\n",
    "    df_train = data.iloc[data.index.difference(df_test.index)] \n",
    "    return df_train.reset_index(drop=True), df_test.reset_index(drop=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.path.join(DATA_DIR, 'chatbot')\n",
    "if not os.path.exists(path):\n",
    "    os.makedirs(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_0, df_test = split_train_test_set_leave_one_out_seq(df, 'user_id', 'timestamp', ['user_id', 'item_id'])\n",
    "df_train, df_valid = split_train_test_set_leave_one_out_seq(df_train_0, 'user_id', 'timestamp', ['user_id', 'item_id',])\n",
    "\n",
    "df_train.to_csv(os.path.join(path, \"train.tsv\"), index=None)\n",
    "df_valid.to_csv(os.path.join(path, \"valid.tsv\"), index=None)\n",
    "df_test.to_csv(os.path.join(path, \"test.tsv\"), index=None)\n",
    "df_train_0.to_csv(os.path.join(path, \"user_history.tsv\"), index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_meta_df = meta_df[meta_df['item_id'].isin(item_map.keys())]\n",
    "saved_meta_df = saved_meta_df.drop_duplicates(subset=['item_id'], keep='first')\n",
    "saved_meta_df.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8678, 6)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saved_meta_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_meta_df['item_id'] = saved_meta_df['item_id'].apply(lambda x: item_map[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>category</th>\n",
       "      <th>price</th>\n",
       "      <th>description</th>\n",
       "      <th>brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5660</td>\n",
       "      <td>Xtreme Brite Brightening Gel 1oz.</td>\n",
       "      <td>[Beauty, Hair Care, Styling Products, Creams, ...</td>\n",
       "      <td>19.99</td>\n",
       "      <td>Xtreme Brite  Brightening gel is a highly conc...</td>\n",
       "      <td>Xtreme Brite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8388</td>\n",
       "      <td>Versace Bright Crystal Eau de Toilette Spray f...</td>\n",
       "      <td>[Beauty, Fragrance, Women's, Eau de Toilette]</td>\n",
       "      <td>52.33</td>\n",
       "      <td>Versace Bright Crystal Perfume for Women 3 oz ...</td>\n",
       "      <td>Versace</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8598</td>\n",
       "      <td>Avalon Biotin B-Complex Thickening Conditioner...</td>\n",
       "      <td>[Beauty, Hair Care, Conditioners]</td>\n",
       "      <td>9.49</td>\n",
       "      <td>It's in our name: Avalon Organics. Our commitm...</td>\n",
       "      <td>Avalon Organics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5325</td>\n",
       "      <td>Better Living Classic Two Chamber Dispenser, W...</td>\n",
       "      <td>[Beauty, Bath &amp; Body, Bathing Accessories, Bat...</td>\n",
       "      <td>25.99</td>\n",
       "      <td>Qality designed two chamber that utilizes a 'p...</td>\n",
       "      <td>Classic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8532</td>\n",
       "      <td>Better Living The Ulti-Mate Dispenser</td>\n",
       "      <td>[Beauty, Bath &amp; Body, Bathing Accessories, Bat...</td>\n",
       "      <td>39.95</td>\n",
       "      <td>The Ulti-Mate Dispenser III, provides a perfec...</td>\n",
       "      <td>Better Living</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8673</th>\n",
       "      <td>7499</td>\n",
       "      <td>Perfect Sleep Mask with Ear Plugs - Ideal Eye ...</td>\n",
       "      <td>[Beauty, Skin Care, Eyes, Masks]</td>\n",
       "      <td>10.75</td>\n",
       "      <td>Do You Want To Improve Your Sleep? Then get th...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8674</th>\n",
       "      <td>2590</td>\n",
       "      <td>Phytoceramides Anti Aging Supplement Reviews -...</td>\n",
       "      <td>[Beauty, Skin Care]</td>\n",
       "      <td>18.36</td>\n",
       "      <td>Phytoceramides:Have you ever wanted to look in...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8675</th>\n",
       "      <td>5016</td>\n",
       "      <td>Dr Song Rosehip Oil 4oz (4 oz)</td>\n",
       "      <td>[Beauty, Skin Care, Face, Oils &amp; Serums]</td>\n",
       "      <td>19.99</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8676</th>\n",
       "      <td>2992</td>\n",
       "      <td>VITAMIN C SERUM 20% with Hyaluronic Acid For Y...</td>\n",
       "      <td>[Beauty, Skin Care, Face, Creams &amp; Moisturizer...</td>\n",
       "      <td>36.00</td>\n",
       "      <td>The Secret to Younger Looking Skin Vitamin C S...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8677</th>\n",
       "      <td>7498</td>\n",
       "      <td>100% Organic Raw Unrefined African Shea Butter...</td>\n",
       "      <td>[Beauty, Skin Care, Body, Moisturizers, Lotions]</td>\n",
       "      <td>9.97</td>\n",
       "      <td>Ingredients:\\n100% Pure, Unrefined, Organic Af...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8678 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id                                              title  \\\n",
       "0        5660                  Xtreme Brite Brightening Gel 1oz.   \n",
       "1        8388  Versace Bright Crystal Eau de Toilette Spray f...   \n",
       "2        8598  Avalon Biotin B-Complex Thickening Conditioner...   \n",
       "3        5325  Better Living Classic Two Chamber Dispenser, W...   \n",
       "4        8532              Better Living The Ulti-Mate Dispenser   \n",
       "...       ...                                                ...   \n",
       "8673     7499  Perfect Sleep Mask with Ear Plugs - Ideal Eye ...   \n",
       "8674     2590  Phytoceramides Anti Aging Supplement Reviews -...   \n",
       "8675     5016                     Dr Song Rosehip Oil 4oz (4 oz)   \n",
       "8676     2992  VITAMIN C SERUM 20% with Hyaluronic Acid For Y...   \n",
       "8677     7498  100% Organic Raw Unrefined African Shea Butter...   \n",
       "\n",
       "                                               category  price  \\\n",
       "0     [Beauty, Hair Care, Styling Products, Creams, ...  19.99   \n",
       "1         [Beauty, Fragrance, Women's, Eau de Toilette]  52.33   \n",
       "2                     [Beauty, Hair Care, Conditioners]   9.49   \n",
       "3     [Beauty, Bath & Body, Bathing Accessories, Bat...  25.99   \n",
       "4     [Beauty, Bath & Body, Bathing Accessories, Bat...  39.95   \n",
       "...                                                 ...    ...   \n",
       "8673                   [Beauty, Skin Care, Eyes, Masks]  10.75   \n",
       "8674                                [Beauty, Skin Care]  18.36   \n",
       "8675           [Beauty, Skin Care, Face, Oils & Serums]  19.99   \n",
       "8676  [Beauty, Skin Care, Face, Creams & Moisturizer...  36.00   \n",
       "8677   [Beauty, Skin Care, Body, Moisturizers, Lotions]   9.97   \n",
       "\n",
       "                                            description            brand  \n",
       "0     Xtreme Brite  Brightening gel is a highly conc...     Xtreme Brite  \n",
       "1     Versace Bright Crystal Perfume for Women 3 oz ...          Versace  \n",
       "2     It's in our name: Avalon Organics. Our commitm...  Avalon Organics  \n",
       "3     Qality designed two chamber that utilizes a 'p...          Classic  \n",
       "4     The Ulti-Mate Dispenser III, provides a perfec...    Better Living  \n",
       "...                                                 ...              ...  \n",
       "8673  Do You Want To Improve Your Sleep? Then get th...              NaN  \n",
       "8674  Phytoceramides:Have you ever wanted to look in...              NaN  \n",
       "8675                                                NaN              NaN  \n",
       "8676  The Secret to Younger Looking Skin Vitamin C S...              NaN  \n",
       "8677  Ingredients:\\n100% Pure, Unrefined, Organic Af...              NaN  \n",
       "\n",
       "[8678 rows x 6 columns]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saved_meta_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_history = df_train_0.groupby('user_id').agg(list)\n",
    "item_count = pd.value_counts(user_history['item_id'])\n",
    "saved_meta_df['visited_num'] = saved_meta_df['id'].apply(lambda x: item_count.loc[x] if x in item_count else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_meta_df.rename(columns={'item_id': 'id'}, inplace=True)\n",
    "saved_meta_df.to_feather(os.path.join(path, 'products.ftr'))\n",
    "saved_meta_df.to_csv(os.path.join(path, 'products.csv'), index=None, sep='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(os.path.join(path, \"train.tsv\"))\n",
    "df_valid = pd.read_csv(os.path.join(path, \"valid.tsv\"))\n",
    "df_test = pd.read_csv(os.path.join(path, \"test.tsv\"))\n",
    "df_train_0 = pd.read_csv(os.path.join(path, \"user_history.tsv\"))\n",
    "saved_meta_df = pd.read_feather(os.path.join(path, 'products.ftr'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data for Simulator\n",
    "\n",
    "user_history = df_train_0.groupby('user_id').agg(list)\n",
    "saved_meta_df = saved_meta_df.set_index('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_title_len = 50\n",
    "id2title = {id: saved_meta_df.loc[id].title[: max_title_len] for id in saved_meta_df.index}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 900\n",
    "test_data = df_test.sample(N, random_state=2024)\n",
    "max_len = 10\n",
    "test_data['history'] = test_data['user_id'].apply(lambda x: '; '.join([id2title[_] for _ in user_history.loc[x]['item_id'][-max_len:]]))\n",
    "test_data['target'] = test_data['item_id'].apply(lambda x: saved_meta_df.loc[x].title)\n",
    "test_data.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       5\n",
       "1      10\n",
       "2       9\n",
       "3       4\n",
       "4       9\n",
       "       ..\n",
       "895     6\n",
       "896    10\n",
       "897    10\n",
       "898     5\n",
       "899     4\n",
       "Name: history, Length: 900, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data['history'].apply(lambda x: len(x.split(\"; \")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "518"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(test_data['history'].apply(len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import *\n",
    "import json, pickle\n",
    "\n",
    "def write_jsonl(obj: List[Dict], fpath: str) -> None:\n",
    "    try:\n",
    "        with open(fpath, 'w') as outfile:\n",
    "            for entry in obj:\n",
    "                json.dump(entry, outfile)\n",
    "                outfile.write('\\n')\n",
    "        print(\"Sucessfully saved into {}.\".format(fpath))\n",
    "    except Exception as e:\n",
    "        print(f\"Error {e} raised. The temp file would be saved in {fpath}.pkl\")\n",
    "        with open(f\"{fpath}.pkl\", 'wb') as tempfile:\n",
    "            pickle.dump(obj, tempfile)\n",
    "    return\n",
    "\n",
    "test_data_jsonl = test_data[['history', 'target']].to_dict(\"records\")\n",
    "write_jsonl(test_data_jsonl, os.path.expanduser(\"~/data/beauty/simulator_test_data_900.jsonl\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "huangxu",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "00b1dd419b15312c24e62f970076858b096939d9edb922fd6f2369a97170d406"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
